<?php
    //echo " Welcome to SK..!!!!"."</br>";
    
    require "policy.php";
    ini_set('memory_limit', '-1');
    require "database_queries.php";
    
    
    function webcrawl($link, $course_selected ){
    
        $valid_url = array();
        
        foreach($link as $url){
            
            
            $html = file_get_contents($url);
            $dom_object = new DOMDocument();
            @$dom_object->loadHTML($html);
            $xpath_node = new DOMXPath($dom_object);
            $hrefs_links = $xpath_node->evaluate("/html/body//a");
            $nested_url = array();
            
            for ($count = 0; $count < $hrefs_links->length; $count++) {
                $href = $hrefs_links->item($count);
                $url_name = $href->getAttribute('href');
                $nested_url[] = $url_name;
                
            }
        
            $good_urls = preg_grep( "/\b(?:(?:https?|ftp):\/\/|www\.)[-a-z0-9+&@#\/%?=~_|!:,.;]*[-a-z0-9+&@#\/%=~_|]/i", $nested_url);
           //$good_urls = array_splice($good_urls,30,30);
            
           //$print_array = implode ( "</br>", $good_urls);
           //echo $print_array;
            $good_urls = array_unique($good_urls);
                        
            foreach ( $good_urls as $url){
                
                if ((validate($url,$course_selected))==true){
                    
                    $a = strpos($url,'blog');
                    $b = strpos($url,$course_selected);
                    
                    if ($a==false){
                        
                        $valid_url[]=$url;
                        echo $url."</br>";
                    }
                    
                    
                    
                    
                }
                
            }
            

        }
        
            
            //$print_array = implode ( "</br>", $valid_url);
           // echo "valid urls are .".$print_array;
        
        
        foreach ($valid_url as $url){
            insertIntoDatabase($course_selected,$url);
        }
            

        
        
        
        
    }
    
    

    
    
?>
        

